#!/usr/bin/python
"""
extract and convert all sgml tree file in one directory
usage: cmd dir
"""

import sys, os, codecs

files=os.listdir(sys.argv[1])
files.sort()

lines = []
for f in files:
	fin = codecs.open(sys.argv[1]+"/"+f,'r',"gb18030")
	ls = fin.readlines()
	fin.close()

	for l in ls:
		if len(l.strip()) == 0:
			continue
		if l.strip()[0] == '<' and l.strip()[-1] == '>':
			continue
		lines.append(l.strip()+"\n")
fs1="".join(lines)
sys.stderr.write(str(len(lines))+"\n")

depth=0
chars = []
for c in fs1:
	if c =='(':
		depth += 1
	if c == ')':
		depth -= 1
	chars.append(c)
	if depth == 0 and c == ')':
		chars.append('\n')
fs2 = "".join(chars)
#sys.stdout.write(fs2)

sents=fs2.split('\n\n')
sys.stderr.write(str(len(sents))+"\n")
treelines=[]
for sen in sents:
	newsen=sen.replace("\n"," ")
	while True:
		newsen = newsen.replace("  "," ")
		newsen = newsen.replace("\t"," ")
		#sys.stderr.write(newsen)
		if newsen.find("  ")<0:
			break
	treelines.append(newsen)
	sys.stdout.write(newsen.encode("utf-8")+"\n")

